package main

import (
	"log"
	"spider"
	. "spider/common"
	"time"
)

func MyConfig(config *SpiderConfig) {
	config.ENABLE_BLOOM_FILTER = true
	config.CONCURRENT_REQUEST = 3
	config.TIMEOUT = time.Second * 8
	config.ENABLE_RETRY = true
	config.ENABLE_PROXY = true
}
func MyHandler(req *Request) {
	req.SetBasicAuth("username", "password")
}
func MyProcesser(resp *Response) (error, *PageItem) {
	page, err := resp.ContentPage()
	if err != nil {
		return err, nil
	}
	var text string
	text, err = page.String()
	if err != nil {
		return err, nil
	}
	log.Println(text[:100])
	resp.AddRequest(resp.Request.URL.String(), nil)
	resp.AddDownload("http://127.0.0.1/1.jpg")
	return nil, nil
}

func main() {

	spider.SpiderName = "CrazySpider"
	spider.StartUrls = []string{
		"http://127.0.0.1",
		"www.baidu.com/./a/b",
	}
	mySpider := spider.NewCrawlSpider()
	mySpider.InitConfig(MyConfig)
	mySpider.HandleRequest(MyHandler)
	mySpider.Process(MyProcesser)
	mySpider.Crawl()
}
